package project3;

import java.util.ArrayList;
import java.util.List;

/** Tokenizes the text of a document into the set of words
 * that appear in the document.
 *
 * Performs tokenization by finding a sequence of characters surrounded by whitespace,
 * then removing all punctuation and casing from the substring.
 *
 * Each token is down cased on return from the tokenizer.
 */
public class Tokenizer implements project3.WordTokenizer
{
    /** Breaks an input text into Word Tokens
     * @param bodyOfText Input text document
     * @return The list of words that appear in the document.
     */
    public List<String> tokenize(String bodyOfText)
    {
        List<String> tokens=new ArrayList<String>();
        for(int at=0; at<bodyOfText.length(); at++)
        {
            //if we hit the first char in a word
            if(!Character.isWhitespace(bodyOfText.charAt(at)))
            {
                //find the last character in the word
                int startIndex=at;
                while(at<bodyOfText.length() && !Character.isWhitespace(bodyOfText.charAt(at)))
                    at++;
                //add the word to the token list
                String token=stripPunctuationAndDownCase(bodyOfText.substring(startIndex, at));
                tokens.add(token);
            }
        }
        return tokens;
    }

    /** Strips all non alpha numeric characters; returning
     * a lower case word.
     * @param text The full input document to strip and downcase
     * @return A whitespace delimited list of words.
     */
    private String stripPunctuationAndDownCase(String text)
    {
        StringBuilder output=new StringBuilder();
        for(int a=0; a<text.length(); a++)
        {
            char currentChar=text.charAt(a);
            //if char is a letter, or digit append it;
            //  otherwise discard it
            if(     Character.isLetter(currentChar)||
                    Character.isDigit(currentChar))
                output.append(Character.toLowerCase(currentChar));
        }
        return output.toString();
    }
}
